package stardust.product.xspider;


import java.io.IOException;
import java.net.URL;
import stardust.core.log.Log;
import stardust.core.network.protocol.http.HttpClient;
import stardust.core.network.protocol.http.HttpMethod;
import stardust.core.network.protocol.http.HttpResponseMeta;
import stardust.product.xspider.support.HTMLSupport;


public class XSpiderProcessor implements Runnable
{
  
  public static final Log log = new Log(XSpiderProcessor.class);
  
  private XSpiderCallback callback;
  
  private HttpClient client = new HttpClient();
  
  public XSpiderProcessor(XSpiderCallback callback)
  {
    this.callback = callback;
  }
  
  public void run()
  {
    while (!Thread.interrupted())
    {
      try
      {
        this.client = new HttpClient();
        
        if (!this.process())
        {
          Thread.sleep(5000);
        }
      }
      catch (InterruptedException e)
      {
        return;
      }
      catch (Throwable e)
      {
        ;
      }
      finally
      {
        this.client.disconnect();
      }
    }
  }

  public boolean process() throws Throwable
  {
    XSpiderURL url = this.callback.nextURL();
    
    if (url == null)
    {
      return false;
    }
    
    try
    {
      HttpResponseMeta rspMeta;
      
      if (url.getReferer() == null)
      {
        rspMeta = client.go(url.getUrl(), HttpMethod.get, null);
      }
      else
      {
        client.setReferer(new URL(url.getReferer()));
        rspMeta = client.click(url.getUrl(), HttpMethod.get, null);
      }
  
      /*
       * 判断文件类型和编码。Content-Type:text/html; charset=UTF-8
       */
      boolean isHTML = false;
      String encode = "utf-8";
      
      if (rspMeta.getOthers().containsKey("Content-Type"))
      {
        String contentType = rspMeta.getOthers().get("Content-Type").trim().toLowerCase();
        int encodeOffset;
        
        if (contentType.startsWith("text/html"))
        {
          isHTML = true;
        }
        
        if ((encodeOffset = contentType.indexOf("charset=")) != -1)
        {
          encode = contentType.substring(encodeOffset + "charset=".length());
        }
        else
        {
          encode = "utf-8";
        }
      }
      
      String title = null;
      String html = null;
      
      if (isHTML)
      {
        html = new String(rspMeta.getData(), encode);
        title = HTMLSupport.getTitle(html, encode);
        this.callback.saveURL(url, HTMLSupport.getURL(url.getUrl(), html, encode));
      }
      
      this.callback.resolve(url, this.client.getLastRequest(), rspMeta, encode, title, html);
      
      return true;
    }
    catch (Throwable e)
    {
      log.warning(e, "\t处理失败 -> " + url.getUrl());
      this.callback.onError(url, e);
      throw e;
    }
  }

}
